{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第五周作业，预测用户是否会对某个活动感兴趣"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#导入工具包\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>event</th>\n",
       "      <th>invited</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>interested</th>\n",
       "      <th>not_interested</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3044012</td>\n",
       "      <td>1918771225</td>\n",
       "      <td>0</td>\n",
       "      <td>2012-10-02 15:53:05.754000+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3044012</td>\n",
       "      <td>1502284248</td>\n",
       "      <td>0</td>\n",
       "      <td>2012-10-02 15:53:05.754000+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3044012</td>\n",
       "      <td>2529072432</td>\n",
       "      <td>0</td>\n",
       "      <td>2012-10-02 15:53:05.754000+00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3044012</td>\n",
       "      <td>3072478280</td>\n",
       "      <td>0</td>\n",
       "      <td>2012-10-02 15:53:05.754000+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3044012</td>\n",
       "      <td>1390707377</td>\n",
       "      <td>0</td>\n",
       "      <td>2012-10-02 15:53:05.754000+00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      user       event  invited                         timestamp  interested  \\\n",
       "0  3044012  1918771225        0  2012-10-02 15:53:05.754000+00:00           0   \n",
       "1  3044012  1502284248        0  2012-10-02 15:53:05.754000+00:00           0   \n",
       "2  3044012  2529072432        0  2012-10-02 15:53:05.754000+00:00           1   \n",
       "3  3044012  3072478280        0  2012-10-02 15:53:05.754000+00:00           0   \n",
       "4  3044012  1390707377        0  2012-10-02 15:53:05.754000+00:00           0   \n",
       "\n",
       "   not_interested  \n",
       "0               0  \n",
       "1               0  \n",
       "2               0  \n",
       "3               0  \n",
       "4               0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#读取数据  train.csv\n",
    "file = open('C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv')\n",
    "train = pd.read_csv(file)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 15398 entries, 0 to 15397\n",
      "Data columns (total 6 columns):\n",
      "user              15398 non-null int64\n",
      "event             15398 non-null int64\n",
      "invited           15398 non-null int64\n",
      "timestamp         15398 non-null object\n",
      "interested        15398 non-null int64\n",
      "not_interested    15398 non-null int64\n",
      "dtypes: int64(5), object(1)\n",
      "memory usage: 721.9+ KB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>event</th>\n",
       "      <th>invited</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1776192</td>\n",
       "      <td>2877501688</td>\n",
       "      <td>0</td>\n",
       "      <td>2012-11-30 11:39:01.230000+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1776192</td>\n",
       "      <td>3025444328</td>\n",
       "      <td>0</td>\n",
       "      <td>2012-11-30 11:39:01.230000+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1776192</td>\n",
       "      <td>4078218285</td>\n",
       "      <td>0</td>\n",
       "      <td>2012-11-30 11:39:01.230000+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1776192</td>\n",
       "      <td>1024025121</td>\n",
       "      <td>0</td>\n",
       "      <td>2012-11-30 11:39:01.230000+00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1776192</td>\n",
       "      <td>2972428928</td>\n",
       "      <td>0</td>\n",
       "      <td>2012-11-30 11:39:21.985000+00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      user       event  invited                         timestamp\n",
       "0  1776192  2877501688        0  2012-11-30 11:39:01.230000+00:00\n",
       "1  1776192  3025444328        0  2012-11-30 11:39:01.230000+00:00\n",
       "2  1776192  4078218285        0  2012-11-30 11:39:01.230000+00:00\n",
       "3  1776192  1024025121        0  2012-11-30 11:39:01.230000+00:00\n",
       "4  1776192  2972428928        0  2012-11-30 11:39:21.985000+00:00"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#读取数据  test.csv\n",
    "file = open('C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/test.csv')\n",
    "test = pd.read_csv(file)\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>locale</th>\n",
       "      <th>birthyear</th>\n",
       "      <th>gender</th>\n",
       "      <th>joinedAt</th>\n",
       "      <th>location</th>\n",
       "      <th>timezone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3197468391</td>\n",
       "      <td>id_ID</td>\n",
       "      <td>1993</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-10-02T06:40:55.524Z</td>\n",
       "      <td>Medan  Indonesia</td>\n",
       "      <td>480.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3537982273</td>\n",
       "      <td>id_ID</td>\n",
       "      <td>1992</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-09-29T18:03:12.111Z</td>\n",
       "      <td>Medan  Indonesia</td>\n",
       "      <td>420.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>823183725</td>\n",
       "      <td>en_US</td>\n",
       "      <td>1975</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-10-06T03:14:07.149Z</td>\n",
       "      <td>Stratford  Ontario</td>\n",
       "      <td>-240.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1872223848</td>\n",
       "      <td>en_US</td>\n",
       "      <td>1991</td>\n",
       "      <td>female</td>\n",
       "      <td>2012-11-04T08:59:43.783Z</td>\n",
       "      <td>Tehran  Iran</td>\n",
       "      <td>210.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3429017717</td>\n",
       "      <td>id_ID</td>\n",
       "      <td>1995</td>\n",
       "      <td>female</td>\n",
       "      <td>2012-09-10T16:06:53.132Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>420.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      user_id locale birthyear  gender                  joinedAt  \\\n",
       "0  3197468391  id_ID      1993    male  2012-10-02T06:40:55.524Z   \n",
       "1  3537982273  id_ID      1992    male  2012-09-29T18:03:12.111Z   \n",
       "2   823183725  en_US      1975    male  2012-10-06T03:14:07.149Z   \n",
       "3  1872223848  en_US      1991  female  2012-11-04T08:59:43.783Z   \n",
       "4  3429017717  id_ID      1995  female  2012-09-10T16:06:53.132Z   \n",
       "\n",
       "             location  timezone  \n",
       "0    Medan  Indonesia     480.0  \n",
       "1    Medan  Indonesia     420.0  \n",
       "2  Stratford  Ontario    -240.0  \n",
       "3        Tehran  Iran     210.0  \n",
       "4                 NaN     420.0  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#读取数据  user.csv\n",
    "file = open('C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/users.csv')\n",
    "users = pd.read_csv(file)\n",
    "users.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>start_time</th>\n",
       "      <th>city</th>\n",
       "      <th>state</th>\n",
       "      <th>zip</th>\n",
       "      <th>country</th>\n",
       "      <th>lat</th>\n",
       "      <th>lng</th>\n",
       "      <th>c_1</th>\n",
       "      <th>...</th>\n",
       "      <th>c_92</th>\n",
       "      <th>c_93</th>\n",
       "      <th>c_94</th>\n",
       "      <th>c_95</th>\n",
       "      <th>c_96</th>\n",
       "      <th>c_97</th>\n",
       "      <th>c_98</th>\n",
       "      <th>c_99</th>\n",
       "      <th>c_100</th>\n",
       "      <th>c_other</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>684921758</td>\n",
       "      <td>3647864012</td>\n",
       "      <td>2012-10-31T00:00:00.001Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>244999119</td>\n",
       "      <td>3476440521</td>\n",
       "      <td>2012-11-03T00:00:00.001Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3928440935</td>\n",
       "      <td>517514445</td>\n",
       "      <td>2012-11-05T00:00:00.001Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2582345152</td>\n",
       "      <td>781585781</td>\n",
       "      <td>2012-10-30T00:00:00.001Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1051165850</td>\n",
       "      <td>1016098580</td>\n",
       "      <td>2012-09-27T00:00:00.001Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 110 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     event_id     user_id                start_time city state  zip country  \\\n",
       "0   684921758  3647864012  2012-10-31T00:00:00.001Z  NaN   NaN  NaN     NaN   \n",
       "1   244999119  3476440521  2012-11-03T00:00:00.001Z  NaN   NaN  NaN     NaN   \n",
       "2  3928440935   517514445  2012-11-05T00:00:00.001Z  NaN   NaN  NaN     NaN   \n",
       "3  2582345152   781585781  2012-10-30T00:00:00.001Z  NaN   NaN  NaN     NaN   \n",
       "4  1051165850  1016098580  2012-09-27T00:00:00.001Z  NaN   NaN  NaN     NaN   \n",
       "\n",
       "   lat  lng  c_1   ...     c_92  c_93  c_94  c_95  c_96  c_97  c_98  c_99  \\\n",
       "0  NaN  NaN    2   ...        0     1     0     0     0     0     0     0   \n",
       "1  NaN  NaN    2   ...        0     0     0     0     0     0     0     0   \n",
       "2  NaN  NaN    0   ...        0     0     0     0     0     0     0     0   \n",
       "3  NaN  NaN    1   ...        0     0     0     0     0     0     0     0   \n",
       "4  NaN  NaN    1   ...        0     0     0     0     0     0     0     0   \n",
       "\n",
       "   c_100  c_other  \n",
       "0      0        9  \n",
       "1      0        7  \n",
       "2      0       12  \n",
       "3      0        8  \n",
       "4      0        9  \n",
       "\n",
       "[5 rows x 110 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#读取数据  events.csv\n",
    "file = open('C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/events.csv')\n",
    "events = pd.read_csv(file)\n",
    "events.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event</th>\n",
       "      <th>yes</th>\n",
       "      <th>maybe</th>\n",
       "      <th>invited</th>\n",
       "      <th>no</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1159822043</td>\n",
       "      <td>1975964455 252302513 4226086795 3805886383 142...</td>\n",
       "      <td>2733420590 517546982 1350834692 532087573 5831...</td>\n",
       "      <td>1723091036 3795873583 4109144917 3560622906 31...</td>\n",
       "      <td>3575574655 1077296663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>686467261</td>\n",
       "      <td>2394228942 2686116898 1056558062 3792942231 41...</td>\n",
       "      <td>1498184352 645689144 3770076778 331335845 4239...</td>\n",
       "      <td>1788073374 733302094 1830571649 676508092 7081...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1186208412</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3320380166 3810793697</td>\n",
       "      <td>1379121209 440668682</td>\n",
       "      <td>1728988561 2950720854</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2621578336</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>855842686</td>\n",
       "      <td>2406118796 3550897984 294255260 1125817077 109...</td>\n",
       "      <td>2671721559 1761448345 2356975806 2666669465 10...</td>\n",
       "      <td>1518670705 880919237 2326414227 2673818347 332...</td>\n",
       "      <td>3500235232</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        event                                                yes  \\\n",
       "0  1159822043  1975964455 252302513 4226086795 3805886383 142...   \n",
       "1   686467261  2394228942 2686116898 1056558062 3792942231 41...   \n",
       "2  1186208412                                                NaN   \n",
       "3  2621578336                                                NaN   \n",
       "4   855842686  2406118796 3550897984 294255260 1125817077 109...   \n",
       "\n",
       "                                               maybe  \\\n",
       "0  2733420590 517546982 1350834692 532087573 5831...   \n",
       "1  1498184352 645689144 3770076778 331335845 4239...   \n",
       "2                              3320380166 3810793697   \n",
       "3                                                NaN   \n",
       "4  2671721559 1761448345 2356975806 2666669465 10...   \n",
       "\n",
       "                                             invited                     no  \n",
       "0  1723091036 3795873583 4109144917 3560622906 31...  3575574655 1077296663  \n",
       "1  1788073374 733302094 1830571649 676508092 7081...                    NaN  \n",
       "2                               1379121209 440668682  1728988561 2950720854  \n",
       "3                                                NaN                    NaN  \n",
       "4  1518670705 880919237 2326414227 2673818347 332...             3500235232  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#活动参加者数据\n",
    "#读取数据\n",
    "file = open(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/event_attendees.csv\")\n",
    "event_attendees = pd.read_csv(file)\n",
    "event_attendees.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 24144 entries, 0 to 24143\n",
      "Data columns (total 5 columns):\n",
      "event      24144 non-null int64\n",
      "yes        22160 non-null object\n",
      "maybe      20977 non-null object\n",
      "invited    22322 non-null object\n",
      "no         17485 non-null object\n",
      "dtypes: int64(1), object(4)\n",
      "memory usage: 943.2+ KB\n"
     ]
    }
   ],
   "source": [
    "event_attendees.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>friends</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3197468391</td>\n",
       "      <td>1346449342 3873244116 4226080662 1222907620 54...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3537982273</td>\n",
       "      <td>1491560444 395798035 2036380346 899375619 3534...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>823183725</td>\n",
       "      <td>1484954627 1950387873 1652977611 4185960823 42...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1872223848</td>\n",
       "      <td>83361640 723814682 557944478 1724049724 253059...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3429017717</td>\n",
       "      <td>4253303705 2130310957 1838389374 3928735761 71...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         user                                            friends\n",
       "0  3197468391  1346449342 3873244116 4226080662 1222907620 54...\n",
       "1  3537982273  1491560444 395798035 2036380346 899375619 3534...\n",
       "2   823183725  1484954627 1950387873 1652977611 4185960823 42...\n",
       "3  1872223848  83361640 723814682 557944478 1724049724 253059...\n",
       "4  3429017717  4253303705 2130310957 1838389374 3928735761 71..."
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#用户友好数据\n",
    "file= open(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/user_friends.csv\")\n",
    "user_friends = pd.read_csv(file)\n",
    "user_friends.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 38202 entries, 0 to 38201\n",
      "Data columns (total 2 columns):\n",
      "user       38202 non-null int64\n",
      "friends    38063 non-null object\n",
      "dtypes: int64(1), object(1)\n",
      "memory usage: 597.0+ KB\n"
     ]
    }
   ],
   "source": [
    "user_friends.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy.sparse as ss\n",
    "import scipy.io as sio\n",
    "\n",
    "#保存数据\n",
    "import pickle\n",
    "from sklearn.preprocessing import normalize\n",
    "#距离\n",
    "import scipy.spatial.distance as ssd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 特征编码工具\n",
    "#该事件涉及国家、城市、时间等信息的处理\n",
    "\n",
    "#保存数据\n",
    "import pickle\n",
    "\n",
    "#特征编码\n",
    "import datetime\n",
    "import hashlib\n",
    "# POSIX locale database and functionality\n",
    "import locale\n",
    "#国家的编码、名字、语言、货币等信息\n",
    "import pycountry\n",
    "\n",
    "import numpy as np\n",
    "import scipy.io as sio\n",
    "import scipy.sparse as ss\n",
    "\n",
    "from collections import defaultdict\n",
    "from sklearn.preprocessing import normalize\n",
    "\n",
    "#类别型特征编码\n",
    "#这里写成类的形式，因为编码字典要在不同的文件中使用\n",
    "class FeatureEng:\n",
    "  def __init__(self):\n",
    "    \n",
    "    # 载入 locales\n",
    "    self.localeIdMap = defaultdict(int)\n",
    "    for i, l in enumerate(locale.locale_alias.keys()):\n",
    "      self.localeIdMap[l] = i + 1\n",
    "    #print locale.locale_alias.keys()\n",
    "\n",
    "    # 载入 countries\n",
    "    self.countryIdMap = defaultdict(int)\n",
    "    ctryIdx = defaultdict(int)\n",
    "    for i, c in enumerate(pycountry.countries):\n",
    "      self.countryIdMap[c.name.lower()] = i + 1\n",
    "      if c.name.lower() == \"usa\":\n",
    "        ctryIdx[\"US\"] = i\n",
    "      if c.name.lower() == \"canada\":\n",
    "        ctryIdx[\"CA\"] = i\n",
    "    for cc in ctryIdx.keys():\n",
    "      for s in pycountry.subdivisions.get(country_code=cc):\n",
    "        self.countryIdMap[s.name.lower()] = ctryIdx[cc] + 1\n",
    "        \n",
    "    # 载入 gender id 字典\n",
    "    ##缺失补0，性别未知\n",
    "    self.genderIdMap = defaultdict(int, {'NaN': 0, \"male\":1, \"female\":2})\n",
    "\n",
    "  def getLocaleId(self, locstr):\n",
    "    return self.localeIdMap[locstr.lower()]\n",
    "\n",
    "  def getGenderId(self, genderStr):\n",
    "    return self.genderIdMap[genderStr]\n",
    "\n",
    "  def getJoinedYearMonth(self, dateString):\n",
    "    try:\n",
    "        dttm = datetime.datetime.strptime(dateString, \"%Y-%m-%dT%H:%M:%S.%fZ\")\n",
    "        return (dttm.year-2010)*12 + dttm.month\n",
    "    except:  #缺失补0\n",
    "        return 0\n",
    "\n",
    "  def getCountryId(self, location):\n",
    "    if (isinstance(location, str)\n",
    "        and len(location.strip()) > 0\n",
    "        and location.rfind(\"  \") > -1):\n",
    "        return self.countryIdMap[location[location.rindex(\"  \") + 2:].lower()]\n",
    "    else:\n",
    "        return 0\n",
    "\n",
    "  def getBirthYearInt(self, birthYear):\n",
    "    try:\n",
    "      return 0 if birthYear == \"None\" else int(birthYear)\n",
    "    except:\n",
    "      return 0\n",
    "\n",
    "  def getTimezoneInt(self, timezone):\n",
    "    try:\n",
    "      return int(timezone)\n",
    "    except:\n",
    "      return 0\n",
    "\n",
    "  def getFeatureHash(self, value):\n",
    "    if len(value.strip()) == 0:\n",
    "      return -1\n",
    "    else:\n",
    "      return int(hashlib.sha224(value.encode('utf8')).hexdigest()[0:4], 16)\n",
    "\n",
    "  def getFloatValue(self, value):\n",
    "    if len(value.strip()) == 0:\n",
    "      return 0.0\n",
    "    else:\n",
    "      return float(value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of records :3137972\n"
     ]
    }
   ],
   "source": [
    "#读取数据，并统计有多少不同的events\n",
    "#其实EDA.ipynb中用read_csv已经统计过了\n",
    "lines = 0\n",
    "fin = open(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/events.csv\", 'rb')\n",
    "#找到用C/C++的感觉了\n",
    "#字段：event_id, user_id,start_time, city, state, zip, country, lat, and lng， 101 columns of words count\n",
    "fin.readline() # skip header，列名行\n",
    "for line in fin:\n",
    "    cols = line.decode('utf8').strip().split(\",\")\n",
    "    lines += 1\n",
    "fin.close()\n",
    "\n",
    "print(\"number of records :%d\" % lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of events in train & test :13418\n"
     ]
    }
   ],
   "source": [
    "#读取训练集和测试集中出现过的活动列表\n",
    "eventIndex = pickle.load(open(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/PE_eventIndex.pkl\", 'rb'))\n",
    "n_events = len(eventIndex)\n",
    "\n",
    "print(\"number of events in train & test :%d\" % n_events)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\scipy\\spatial\\distance.py:698: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  dist = 1.0 - uv / np.sqrt(uu * vv)\n"
     ]
    }
   ],
   "source": [
    "FE = FeatureEng()\n",
    "\n",
    "fin = open(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/events.csv\", 'rb')\n",
    "\n",
    "#字段：event_id, user_id,start_time, city, state, zip, country, lat, and lng， 101 columns of words count\n",
    "fin.readline() # skip header\n",
    "\n",
    "#start_time, city, state, zip, country, lat, and lng\n",
    "eventPropMatrix = ss.dok_matrix((n_events, 7))\n",
    "\n",
    "#词频特征\n",
    "eventContMatrix = ss.dok_matrix((n_events, 101))\n",
    "\n",
    "for line in fin.readlines():\n",
    "    cols = line.decode('utf8').strip().split(\",\")\n",
    "    eventId = str(cols[0])\n",
    "    \n",
    "    if eventId in eventIndex:  #在训练集或测试集中出现\n",
    "        i = eventIndex[eventId]\n",
    "  \n",
    "        #event的特征编码，这里只是简单处理，其实开始时间，地点等信息很重要\n",
    "        eventPropMatrix[i, 0] = FE.getJoinedYearMonth(cols[2]) # start_time\n",
    "        eventPropMatrix[i, 1] = FE.getFeatureHash(cols[3]) # city\n",
    "        eventPropMatrix[i, 2] = FE.getFeatureHash(cols[4]) # state\n",
    "        eventPropMatrix[i, 3] = FE.getFeatureHash(cols[5]) # zip\n",
    "        eventPropMatrix[i, 4] = FE.getFeatureHash(cols[6]) # country\n",
    "        eventPropMatrix[i, 5] = FE.getFloatValue(cols[7]) # lat\n",
    "        eventPropMatrix[i, 6] = FE.getFloatValue(cols[8]) # lon\n",
    "        \n",
    "        #词频\n",
    "        for j in range(9, 109):\n",
    "            eventContMatrix[i, j-9] = cols[j]\n",
    "fin.close()\n",
    "\n",
    "#用L2模归一化\n",
    "eventPropMatrix = normalize(eventPropMatrix,\n",
    "    norm=\"l2\", copy=False)\n",
    "sio.mmwrite(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/EV_eventPropMatrix\", eventPropMatrix)\n",
    "\n",
    "#词频，可以考虑我们用这部分特征进行聚类，得到活动的genre\n",
    "eventContMatrix = normalize(eventContMatrix,\n",
    "    norm=\"l2\", copy=False)\n",
    "sio.mmwrite(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/EV_eventContMatrix\", eventContMatrix)\n",
    "\n",
    "\n",
    "# calculate 相似度 between event pairs based on the two matrices\n",
    "eventPropSim = ss.dok_matrix((n_events, n_events))\n",
    "eventContSim = ss.dok_matrix((n_events, n_events))\n",
    "\n",
    "    \n",
    "#读取在测试集和训练集中出现的活动对\n",
    "uniqueEventPairs = pickle.load(open(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/PE_uniqueEventPairs.pkl\", 'rb'))\n",
    "\n",
    "for e1, e2 in uniqueEventPairs:\n",
    "    #i = eventIndex[e1]\n",
    "    #j = eventIndex[e2]\n",
    "    i = e1\n",
    "    j = e2\n",
    "    \n",
    "    #非词频特征，correlation相似度 （1 - correlation distance）\n",
    "    #注意：scipy.spatial.distance中correlation返回的是距离，即1-相关系数，取值范围[0,2]\n",
    "    #这里我们用的是相似度，将距离再还原成相似度\n",
    "    if i not in eventPropSim and j not in eventPropSim:\n",
    "        epsim = 1 - ssd.correlation(eventPropMatrix.getrow(i).todense(),\n",
    "            eventPropMatrix.getrow(j).todense())\n",
    "        \n",
    "        eventPropSim[i, j] = epsim\n",
    "        eventPropSim[j, i] = epsim\n",
    "    \n",
    "    #对词频特征，Cosine 相似度（1 - cosnie distance）\n",
    "    if i not in eventContSim and j not in eventContSim:\n",
    "        ec_dist = 1 - ssd.cosine(eventContMatrix.getrow(i).todense(),\n",
    "            eventContMatrix.getrow(j).todense())\n",
    "    \n",
    "        eventContSim[i, j] = ec_dist\n",
    "        eventContSim[j, i] = ec_dist\n",
    "    \n",
    "sio.mmwrite(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/EV_eventPropSim\", eventPropSim)\n",
    "sio.mmwrite(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/EV_eventContSim\", eventContSim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[0., 0., 0., ..., 0., 0., 0.]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eventPropSim.getrow(0).todense()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of users in train & test :3391\n"
     ]
    }
   ],
   "source": [
    "#读取训练集和测试集中出现过的用户列表\n",
    "userIndex = pickle.load(open(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/PE_userIndex.pkl\", 'rb'))\n",
    "n_users = len(userIndex)\n",
    "\n",
    "print(\"number of users in train & test :%d\" % n_users)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>locale</th>\n",
       "      <th>birthyear</th>\n",
       "      <th>gender</th>\n",
       "      <th>joinedAt</th>\n",
       "      <th>location</th>\n",
       "      <th>timezone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3197468391</td>\n",
       "      <td>id_ID</td>\n",
       "      <td>1993</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-10-02T06:40:55.524Z</td>\n",
       "      <td>Medan  Indonesia</td>\n",
       "      <td>480.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3537982273</td>\n",
       "      <td>id_ID</td>\n",
       "      <td>1992</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-09-29T18:03:12.111Z</td>\n",
       "      <td>Medan  Indonesia</td>\n",
       "      <td>420.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>823183725</td>\n",
       "      <td>en_US</td>\n",
       "      <td>1975</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-10-06T03:14:07.149Z</td>\n",
       "      <td>Stratford  Ontario</td>\n",
       "      <td>-240.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1872223848</td>\n",
       "      <td>en_US</td>\n",
       "      <td>1991</td>\n",
       "      <td>female</td>\n",
       "      <td>2012-11-04T08:59:43.783Z</td>\n",
       "      <td>Tehran  Iran</td>\n",
       "      <td>210.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3429017717</td>\n",
       "      <td>id_ID</td>\n",
       "      <td>1995</td>\n",
       "      <td>female</td>\n",
       "      <td>2012-09-10T16:06:53.132Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>420.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      user_id locale birthyear  gender                  joinedAt  \\\n",
       "0  3197468391  id_ID      1993    male  2012-10-02T06:40:55.524Z   \n",
       "1  3537982273  id_ID      1992    male  2012-09-29T18:03:12.111Z   \n",
       "2   823183725  en_US      1975    male  2012-10-06T03:14:07.149Z   \n",
       "3  1872223848  en_US      1991  female  2012-11-04T08:59:43.783Z   \n",
       "4  3429017717  id_ID      1995  female  2012-09-10T16:06:53.132Z   \n",
       "\n",
       "             location  timezone  \n",
       "0    Medan  Indonesia     480.0  \n",
       "1    Medan  Indonesia     420.0  \n",
       "2  Stratford  Ontario    -240.0  \n",
       "3        Tehran  Iran     210.0  \n",
       "4                 NaN     420.0  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#读取数据\n",
    "file = open(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/users.csv\")\n",
    "users = pd.read_csv(file)\n",
    "users.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 38209 entries, 0 to 38208\n",
      "Data columns (total 7 columns):\n",
      "user_id      38209 non-null int64\n",
      "locale       38209 non-null object\n",
      "birthyear    38209 non-null object\n",
      "gender       38100 non-null object\n",
      "joinedAt     38152 non-null object\n",
      "location     32745 non-null object\n",
      "timezone     37773 non-null float64\n",
      "dtypes: float64(1), int64(1), object(5)\n",
      "memory usage: 2.0+ MB\n"
     ]
    }
   ],
   "source": [
    "users.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "FE = FeatureEng()\n",
    "\n",
    "#locale\tbirthyear\tgender\tjoinedAt\tlocation\ttimezone\n",
    "#去掉user_id列\n",
    "n_cols = users.shape[1] - 1\n",
    "cols = ['LocaleId', 'BirthYearInt', 'GenderId', 'JoinedYearMonth', 'CountryId', 'TimezoneInt']\n",
    "\n",
    "#users编码后的特征\n",
    "#userMatrix = np.zeros((n_users, n_cols), dtype=np.int)\n",
    "userMatrix = ss.dok_matrix((n_users, n_cols))\n",
    "\n",
    "for u in range(users.shape[0]): \n",
    "    userId = str(users.loc[u,'user_id'])\n",
    "    \n",
    "    if userId in userIndex:  #在训练集或测试集中出现\n",
    "        i = userIndex[userId]\n",
    "    \n",
    "        userMatrix[i, 0] = FE.getLocaleId(users.loc[u,'locale'])\n",
    "        userMatrix[i, 1] = FE.getBirthYearInt(users.loc[u,'birthyear'])\n",
    "        userMatrix[i, 2] = FE.getGenderId(users.loc[u,'gender'])\n",
    "        userMatrix[i, 3] = FE.getJoinedYearMonth(users.loc[u,'joinedAt'])\n",
    "        \n",
    "        #由于地点的写法不规范，该编码似乎不起作用（所有样本的特征都被编码成0了）\n",
    "        userMatrix[i, 4] = FE.getCountryId(users.loc[u,'location'])\n",
    "        \n",
    "        userMatrix[i, 5] = FE.getTimezoneInt(users.loc[u,'timezone'])\n",
    "\n",
    "# 归一化用户矩阵\n",
    "userMatrix = normalize(userMatrix, norm=\"l2\", copy=False)\n",
    "sio.mmwrite(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/US_userMatrix\", userMatrix)\n",
    "\n",
    "\n",
    "# 计算用户对的相似度矩阵，之后用于推荐系统\n",
    "userSimMatrix = ss.dok_matrix((n_users, n_users))\n",
    "\n",
    "#读取在测试集和训练集中出现的用户对\n",
    "uniqueUserPairs = pickle.load(open(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/FE_uniqueUserPairs.pkl\", 'rb'))\n",
    "\n",
    "#对角线元素\n",
    "for i in range(0, n_users):\n",
    "    userSimMatrix[i, i] = 1.0\n",
    "    \n",
    "#对称\n",
    "for u1, u2 in uniqueUserPairs:\n",
    "    #i = userIndex[u1]\n",
    "    #j = userIndex[u2]\n",
    "    i = u1\n",
    "    j = u2\n",
    "    if i not in userSimMatrix and j not in userSimMatrix:\n",
    "        #Person相关系数做为相似度度量\n",
    "        #特征：国家（locale、location）、年龄、性别、时区、地点\n",
    "        #usim = ssd.correlation(userMatrix[i,:],\n",
    "            #userMatrix[j,:])\n",
    "           \n",
    "        #注意：scipy.spatial.distance中correlation返回的是距离，即1-相关系数，取值范围[0,2]\n",
    "        #这里我们用的是相似度，将距离再还原成相似度\n",
    "        usim = 1 - ssd.correlation(userMatrix.getrow(i).todense(),\n",
    "          userMatrix.getrow(j).todense())\n",
    "        userSimMatrix[i, j] = usim\n",
    "        userSimMatrix[j, i] = usim\n",
    "    \n",
    "sio.mmwrite(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/US_userSimMatrix\", userSimMatrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of users in train & test :3391\n"
     ]
    }
   ],
   "source": [
    "#读取训练集和测试集中出现过的事件列表\n",
    "userIndex = pickle.load(open(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/PE_userIndex.pkl\", 'rb'))\n",
    "n_users = len(userIndex)\n",
    "\n",
    "print(\"number of users in train & test :%d\" % n_users)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#用户-事件关系矩阵\n",
    "userEventScores = sio.mmread(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/PE_userEventScores\")\n",
    "\n",
    "#后续用于将用户朋友参加的活动影响到用户\n",
    "eventsForUser = pickle.load(open(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/PE_eventsForUser.pkl\", 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "#读取数据\n",
    "\n",
    "\"\"\"\n",
    "  找出某用户的那些朋友\n",
    "  1)如果你有更多的朋友，可能你性格外向，更容易参加各种活动\n",
    "  2)如果你朋友会参加某个活动，可能你也会跟随去参加一下\n",
    "\"\"\"\n",
    " \n",
    "#用户有多少个朋友\n",
    "numFriends = np.zeros((n_users))\n",
    "userFriends = ss.dok_matrix((n_users, n_users))\n",
    "    \n",
    "fin = open(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/user_friends.csv\", 'rb')\n",
    "#字段：user，friends\n",
    "fin.readline()                # skip header\n",
    "\n",
    "#ln = 0\n",
    "for line in fin:  #对每个用户        \n",
    "    cols = line.decode('utf8').strip().split(\",\")\n",
    "    user = str(cols[0])    #user\n",
    "    \n",
    "    if user in userIndex:   #该用户在训练集和测试集的用户列表中\n",
    "        friends = cols[1].split(\" \")  #friends\n",
    "        i = userIndex[user]       #该用户的索引\n",
    "        numFriends[i] = len(friends)\n",
    "        for friend in friends:  #该用户的每个朋友\n",
    "            str_friend = str(friend)\n",
    "            if str_friend in userIndex:  #如果朋友也在训练集或测试集中出现\n",
    "                j = userIndex[str_friend]   #朋友的索引\n",
    "            \n",
    "                # the objective of this score is to infer the degree to\n",
    "                # and direction in which this friend will influence the\n",
    "                # user's decision, so we sum the user/event score for\n",
    "                # this user across all training events.\n",
    "            \n",
    "                #userEventScores为用户对活动的打分（interested - not interseted）\n",
    "                #在Users-Events.ipynb中计算好了\n",
    "                eventsForUser = userEventScores.getrow(j).todense()\n",
    "            \n",
    "                #所有朋友参加活动的数量（平均频率）\n",
    "                score = eventsForUser.sum() / np.shape(eventsForUser)[1]\n",
    "                userFriends[i, j] += score\n",
    "                userFriends[j, i] += score\n",
    "            \n",
    "fin.close()\n",
    "    \n",
    "\n",
    "#用户的朋友数目\n",
    "# 归一化数组\n",
    "sumNumFriends = numFriends.sum(axis=0)\n",
    "numFriends = numFriends / sumNumFriends\n",
    "sio.mmwrite(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/UF_numFriends\", np.matrix(numFriends))\n",
    "\n",
    "#\n",
    "userFriends = normalize(userFriends, norm=\"l2\", copy=False)\n",
    "sio.mmwrite(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/UF_userFriends\", userFriends)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([9.85427096e-04, 5.27687232e-04, 1.18749727e-03, ...,\n",
       "       3.42500905e-04, 2.47361765e-04, 3.40356925e-05])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "numFriends"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of events in train & test :13418\n"
     ]
    }
   ],
   "source": [
    "#读取训练集和测试集中出现过的事件列表\n",
    "eventIndex = pickle.load(open(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/PE_eventIndex.pkl\", 'rb'))\n",
    "n_events = len(eventIndex)\n",
    "\n",
    "print(\"number of events in train & test :%d\" % n_events)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "#读取数据\n",
    "\"\"\"\n",
    "  统计某个活动，参加和不参加的人数，计算活动热度\n",
    "\"\"\"\n",
    "\n",
    "#活动活跃度\n",
    "eventPopularity = ss.dok_matrix((n_events, 1))\n",
    "    \n",
    "f = open(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/event_attendees.csv\", 'rb')\n",
    "\n",
    "#字段：event_id,yes, maybe, invited, and no\n",
    "f.readline() # skip header\n",
    "\n",
    "for line in f:\n",
    "    cols = line.decode('utf8').strip().split(\",\")\n",
    "    eventId = str(cols[0])   #event_id\n",
    "    if eventId in eventIndex:\n",
    "        i = eventIndex[eventId]  #事件索引\n",
    "        \n",
    "        #yes - no\n",
    "        eventPopularity[i, 0] = \\\n",
    "          len(cols[1].split(\" \")) - len(cols[4].split(\" \"))\n",
    "    \n",
    "f.close()\n",
    "    \n",
    "eventPopularity = normalize(eventPopularity, norm=\"l1\",\n",
    "     copy=False)\n",
    "sio.mmwrite(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/EA_eventPopularity\", eventPopularity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "matrix([[ 1.],\n",
       "        [ 1.],\n",
       "        [ 0.],\n",
       "        ...,\n",
       "        [ 1.],\n",
       "        [-1.],\n",
       "        [ 1.]])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eventPopularity.todense()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将所有特征串联起来，构成RS_Train.csv\n",
    "#RS_Test.csv\n",
    "#为最后推荐系统做准备\n",
    "\n",
    "import numpy as np\n",
    "import scipy.io as sio\n",
    "import scipy.sparse as ss\n",
    "from numpy.random import random  \n",
    "from collections import defaultdict\n",
    "import scipy.spatial.distance as ssd\n",
    "\n",
    "class RecommonderSystem:\n",
    "  def __init__(self):\n",
    "    # 读入数据做初始化\n",
    "    \n",
    "    #用户和活动新的索引\n",
    "    self.userIndex = pickle.load(open(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/PE_userIndex.pkl\", 'rb'))\n",
    "    self.eventIndex = pickle.load(open(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/PE_eventIndex.pkl\", 'rb'))\n",
    "    self.n_users = len(self.userIndex)\n",
    "    self.n_items = len(self.eventIndex)\n",
    "    \n",
    "    #用户-活动关系矩阵R\n",
    "    #在train_SVD会重新从文件中读取,二者要求的格式不同，来不及统一了:(\n",
    "    self.userEventScores = sio.mmread(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/PE_userEventScores\").todense()\n",
    "    #每个用户的平均打分，用于基于物品的协同过滤中物品相似度度量中去掉用户打分习惯的影响\n",
    "    self.userMeanScore = np.mean(self.userEventScores, axis = 1)\n",
    "   \n",
    "    #倒排表\n",
    "    ##每个用户参加的事件\n",
    "    self.itemsForUser = pickle.load(open(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/PE_eventsForUser.pkl\", 'rb'))\n",
    "    ##事件参加的用户\n",
    "    self.usersForItem = pickle.load(open(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/PE_usersForEvent.pkl\", 'rb'))\n",
    "    \n",
    "    #基于模型的协同过滤参数初始化,训练\n",
    "    self.init_SVD()\n",
    "    self.train_SVD(trainfile = \"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv\")\n",
    "    \n",
    "    #根据用户属性计算出的用户之间的相似度\n",
    "    self.userSimMatrix = sio.mmread(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/US_userSimMatrix\").todense()\n",
    "    \n",
    "    #根据活动属性计算出的活动之间的相似度\n",
    "    self.eventPropSim = sio.mmread(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/EV_eventPropSim\").todense()\n",
    "    self.eventContSim = sio.mmread(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/EV_eventContSim\").todense()\n",
    "    \n",
    "    #每个用户的朋友的数目\n",
    "    self.numFriends = sio.mmread(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/UF_numFriends\")\n",
    "    #用户的每个朋友参加活动的分数对该用户的影响\n",
    "    self.userFriends = sio.mmread(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/UF_userFriends\").todense()\n",
    "    \n",
    "    #活动本身的热度\n",
    "    self.eventPopularity = sio.mmread(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/EA_eventPopularity\").todense()\n",
    "\n",
    "  def init_SVD(self, K=20):\n",
    "    #初始化模型参数（for 基于模型的协同过滤SVD_CF）\n",
    "    self.K = K  \n",
    "    \n",
    "    #init parameters\n",
    "    #bias\n",
    "    self.bi = np.zeros(self.n_items)  \n",
    "    self.bu = np.zeros(self.n_users)  \n",
    "    \n",
    "    #the small matrix\n",
    "    self.P = random((self.n_users,self.K))/10*(np.sqrt(self.K))\n",
    "    self.Q = random((self.K, self.n_items))/10*(np.sqrt(self.K))  \n",
    "                  \n",
    "          \n",
    "  def train_SVD(self,trainfile = 'C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv', steps=100,gamma=0.04,Lambda=0.15):\n",
    "    #训练SVD模型（for 基于模型的协同过滤SVD_CF）\n",
    "    #gamma：为学习率\n",
    "    #Lambda：正则参数\n",
    "    \n",
    "    #偷懒了，为了和原来的代码的输入接口一样，直接从训练文件中去读取数据\n",
    "    print(\"SVD Train...\")\n",
    "    ftrain = open(trainfile, 'r')\n",
    "    ftrain.readline()\n",
    "    self.mu = 0.0\n",
    "    n_records = 0\n",
    "    uids = []  #每条记录的用户索引\n",
    "    i_ids = [] #每条记录的item索引\n",
    "    #用户-Item关系矩阵R（内容同userEventScores相同），临时变量，训练完了R不再需要\n",
    "    R = np.zeros((self.n_users, self.n_items))\n",
    "    \n",
    "    for line in ftrain:\n",
    "        cols = line.strip().split(\",\")\n",
    "        u = self.userIndex[cols[0]]  #用户\n",
    "        i = self.eventIndex[cols[1]] #活动\n",
    "        \n",
    "        uids.append(u)\n",
    "        i_ids.append(i)\n",
    "        \n",
    "        R[u,i] = int(cols[4])  #interested\n",
    "        self.mu += R[u,i]\n",
    "        n_records += 1\n",
    "    \n",
    "    ftrain.close()\n",
    "    self.mu /= n_records\n",
    "    \n",
    "    for step in range(steps):  \n",
    "        #print 'the ',step,'-th  step is running'  \n",
    "        rmse_sum=0.0 \n",
    "            \n",
    "        #将训练样本打散顺序\n",
    "        kk = np.random.permutation(n_records)  \n",
    "        for j in range(n_records):  \n",
    "            #每次一个训练样本\n",
    "            index = kk[j]  \n",
    "            #temp = self.nonzero_scores_index[b]\n",
    "            #u = temp[0]\n",
    "            #i = temp[1]\n",
    "            u = uids[index]\n",
    "            i = i_ids[index]\n",
    " \n",
    "            #预测残差\n",
    "            eui = R[u,i] - self.pred_SVD(u,i)\n",
    "            #残差平方和\n",
    "            rmse_sum += eui**2\n",
    "               \n",
    "            #随机梯度下降，更新\n",
    "            self.bu[u]+= gamma*(eui - Lambda*self.bu[u])  \n",
    "            self.bi[i]+= gamma*(eui - Lambda*self.bi[i]) \n",
    "            \n",
    "            for k in range(self.K):\n",
    "                #P,Q 同时更新，temp暂存P的更新之的值\n",
    "                temp = self.P[u,k] + gamma * eui * self.Q[k,i] - Lambda * self.P[u,k]\n",
    "                #self.P[u,k] += gamma * eui * self.Q[k,i] - Lambda * self.P[u,k]\n",
    "                self.Q[k,i] += gamma * eui * self.P[u,k] - Lambda * self.Q[k,i]\n",
    "                self.P[u,k] = temp\n",
    "                \n",
    "        #学习率递减\n",
    "        gamma=gamma*0.93  \n",
    "        #print(\"the rmse of the {} th step on train data is:{}\".format(step, rmse_sum))\n",
    "    print(\"SVD trained\")\n",
    "    \n",
    "  def pred_SVD(self, uid, i_id):\n",
    "    #根据当前参数，预测用户uid对Item（i_id）的打分        \n",
    "    ans=self.mu + self.bi[i_id] + self.bu[uid] + np.dot(self.P[uid,:],self.Q[:,i_id])  \n",
    "        \n",
    "    #将打分范围控制在0-1之间\n",
    "    if ans>1:  \n",
    "        return 1  \n",
    "    elif ans<0:  \n",
    "        return 0\n",
    "    return ans  \n",
    "\n",
    "  def sim_cal_UserCF(self, uid1, uid2 ):\n",
    "    si={}  #有效item（两个用户均有打分的item）的集合\n",
    "    for item in self.itemsForUser[uid1]:  #uid1所有打过分的Item1\n",
    "        if item in self.itemsForUser[uid2]:  #如果uid2也对该Item打过分\n",
    "            si[item]=1  #item为一个有效item\n",
    "        \n",
    "    #print si\n",
    "    n=len(si)   #有效item数，有效item为即对uid对Item打过分，uid2也对Item打过分\n",
    "    if (n==0):  #没有共同打过分的item，相似度设为0？\n",
    "        similarity=0  \n",
    "        return similarity  \n",
    "        \n",
    "    #用户uid1打过分的所有有效的item\n",
    "    s1=np.array([self.userEventScores[uid1,item] for item in si])  \n",
    "        \n",
    "    #用户uid2打过分的所有有效的Item\n",
    "    s2=np.array([self.userEventScores[uid2,item] for item in si])  \n",
    "        \n",
    "    sum1=np.sum(s1)  \n",
    "    sum2=np.sum(s2)  \n",
    "    sum1Sq=np.sum(s1**2)  \n",
    "    sum2Sq=np.sum(s2**2)  \n",
    "    pSum=np.sum(s1*s2)  \n",
    "        \n",
    "    #分子\n",
    "    num=pSum-(sum1*sum2/n)  \n",
    "        \n",
    "    #分母\n",
    "    den=np.sqrt((sum1Sq-sum1**2/n)*(sum2Sq-sum2**2/n))  \n",
    "    if den==0: #这个数据点不好用，舍弃 \n",
    "        similarity=0  \n",
    "        return 0  \n",
    "        \n",
    "    similarity = num/den  \n",
    "    return similarity  \n",
    "\n",
    "  def userCFReco(self, userId, eventId):\n",
    "    \"\"\"\n",
    "    根据User-based协同过滤，得到event的推荐度\n",
    "    基本的伪代码思路如下：\n",
    "    for item i\n",
    "      for every other user v that has a preference for i\n",
    "        compute similarity s between u and v\n",
    "        incorporate v's preference for i weighted by s into running aversge\n",
    "    return top items ranked by weighted average\n",
    "    \"\"\"\n",
    "    \n",
    "    u = self.userIndex[userId]\n",
    "    i = self.eventIndex[eventId]\n",
    "    \n",
    "    sim_accumulate=0.0  \n",
    "    rat_acc=0.0  \n",
    "\n",
    "    for user in self.usersForItem[i]:  #对eventId打过分的所有用户\n",
    "        #print user, u\n",
    "        sim = self.sim_cal_UserCF(uid1 = user,uid2 = u)    #该user与uid之间的相似度\n",
    "        if sim == 0:continue  \n",
    "            #print sim,self.user_movie[uid][item],sim*self.user_movie[uid][item]  \n",
    "            \n",
    "        #u2 = self.userIndex[user]\n",
    "        rat_acc += sim * self.userEventScores[user,i]   #用户user对eventId的打分\n",
    "        sim_accumulate += sim  \n",
    "        \n",
    "    #print rat_acc,sim_accumulate  \n",
    "    if sim_accumulate==0: #no same user rated,return average rates of the data  \n",
    "        return  self.mu  \n",
    "    ans = rat_acc/sim_accumulate  \n",
    "\n",
    "    #将打分范围控制在0-1之间\n",
    "    if ans>1:  \n",
    "        return 1  \n",
    "    elif ans<0:  \n",
    "        return 0  \n",
    "    return ans\n",
    "\n",
    "\n",
    "  def sim_cal_ItemCF(self, i_id1, i_id2):\n",
    "    #计算Item i_id1和i_id2之间的相似性\n",
    "    si={}  #有效用户集合\n",
    "    for user in self.usersForItem[i_id1]:  #所有对Item1打过分的的user\n",
    "        if user in self.usersForItem[i_id2]:  #如果该用户对Item2也打过分\n",
    "            si[user]=1  #user为一个有效用用户\n",
    "        \n",
    "    n=len(si)   #有效用户数，有效用户为即对Item1打过分，也对Item2打过分\n",
    "    if (n==0):  #没有共同打过分的用户，相似度设为0？\n",
    "        return 0  \n",
    "        \n",
    "    #所有有效用户对Item1的打分\n",
    "    s1=np.array([self.userEventScores[u, i_id1] for u in si])  \n",
    "        \n",
    "    #所有有效用户对Item2的打分\n",
    "    s2=np.array([self.userEventScores[u, i_id2] for u in si])\n",
    "    \n",
    "\n",
    "    #修正有效打分, 减去用户的平均打分\n",
    "    user_mean_score = np.array([self.userMeanScore[u,0] for u in si])\n",
    "    s1 = s1 - user_mean_score;\n",
    "    s2 = s2 - user_mean_score;\n",
    "   \n",
    "    #余弦相似度\n",
    "    sim = 1- ssd.cosine(s1, s2)\n",
    "    return sim  \n",
    "            \n",
    "  def eventCFReco(self, userId, eventId):    \n",
    "    \"\"\"\n",
    "    根据基于物品的协同过滤，得到Event的推荐度\n",
    "    基本的伪代码思路如下：\n",
    "    for item i \n",
    "        for every item j tht u has a preference for\n",
    "            compute similarity s between i and j\n",
    "            add u's preference for j weighted by s to a running average\n",
    "    return top items, ranked by weighted average\n",
    "    \"\"\"\n",
    "    u = self.userIndex[userId]\n",
    "    i = self.eventIndex[eventId]\n",
    "\n",
    "    sim_accumulate=0.0  \n",
    "    rat_acc=0.0  \n",
    "                   \n",
    "    for item in self.itemsForUser[u]:  #用户uid打过分的所有Item\n",
    "        #i2 = self.eventIndex[item]\n",
    "        sim = self.sim_cal_ItemCF(item,i)    #该Item与i_id之间的相似度\n",
    "           \n",
    "        rat_acc += sim * self.userEventScores[u,item]  \n",
    "        sim_accumulate += sim  \n",
    "        \n",
    "    #print rat_acc,sim_accumulate  \n",
    "    if sim_accumulate==0: #no same user rated,return average rates of the data  \n",
    "        return  self.mu  \n",
    "\n",
    "    ans = rat_acc/sim_accumulate  \n",
    "\n",
    "    #将打分范围控制在0-1之间\n",
    "    if ans>1:  \n",
    "        return 1  \n",
    "    elif ans<0:  \n",
    "        return 0\n",
    "    return ans\n",
    "    \n",
    "  def svdCFReco(self, userId, eventId):\n",
    "    #基于模型的协同过滤, SVD++/LFM\n",
    "    u = self.userIndex[userId]\n",
    "    i = self.eventIndex[eventId]\n",
    "\n",
    "    return self.pred_SVD(u,i)\n",
    "\n",
    "  def userReco(self, userId, eventId):\n",
    "    \"\"\"\n",
    "    类似基于User-based协同过滤，只是用户之间的相似度由用户本身的属性得到，计算event的推荐度\n",
    "    基本的伪代码思路如下：\n",
    "    for item i\n",
    "      for every other user v that has a preference for i\n",
    "        compute similarity s between u and v\n",
    "        incorporate v's preference for i weighted by s into running aversge\n",
    "    return top items ranked by weighted average\n",
    "    \"\"\"\n",
    "    i = self.userIndex[userId]\n",
    "    j = self.eventIndex[eventId]\n",
    "\n",
    "    vs = self.userEventScores[:, j]\n",
    "    sims = self.userSimMatrix[i, :]\n",
    "\n",
    "    prod = sims * vs\n",
    "\n",
    "    try:\n",
    "      return prod[0, 0] - self.userEventScores[i, j]\n",
    "    except IndexError:\n",
    "      return 0\n",
    "\n",
    "  def eventReco(self, userId, eventId):\n",
    "    \"\"\"\n",
    "    类似基于Item-based协同过滤，只是item之间的相似度由item本身的属性得到，计算Event的推荐度\n",
    "    基本的伪代码思路如下：\n",
    "    for item i \n",
    "      for every item j that u has a preference for\n",
    "        compute similarity s between i and j\n",
    "        add u's preference for j weighted by s to a running average\n",
    "    return top items, ranked by weighted average\n",
    "    \"\"\"\n",
    "    i = self.userIndex[userId]\n",
    "    j = self.eventIndex[eventId]\n",
    "    js = self.userEventScores[i, :]\n",
    "    psim = self.eventPropSim[:, j]\n",
    "    csim = self.eventContSim[:, j]\n",
    "    pprod = js * psim\n",
    "    cprod = js * csim\n",
    "    \n",
    "    pscore = 0\n",
    "    cscore = 0\n",
    "    try:\n",
    "      pscore = pprod[0, 0] - self.userEventScores[i, j]\n",
    "    except IndexError:\n",
    "      pass\n",
    "    try:\n",
    "      cscore = cprod[0, 0] - self.userEventScores[i, j]\n",
    "    except IndexError:\n",
    "      pass\n",
    "    return pscore, cscore\n",
    "\n",
    "  def userPop(self, userId):\n",
    "    \"\"\"\n",
    "    基于用户的朋友个数来推断用户的社交程度\n",
    "    主要的考量是如果用户的朋友非常多，可能会更倾向于参加各种社交活动\n",
    "    \"\"\"\n",
    "    if userId in self.userIndex:\n",
    "      i = self.userIndex[userId]\n",
    "      try:\n",
    "        return self.numFriends[0, i]\n",
    "      except IndexError:\n",
    "        return 0\n",
    "    else:\n",
    "      return 0\n",
    "\n",
    "  def friendInfluence(self, userId):\n",
    "    \"\"\"\n",
    "    朋友对用户的影响\n",
    "    主要考虑用户所有的朋友中，有多少是非常喜欢参加各种社交活动/event的\n",
    "    用户的朋友圈如果都积极参与各种event，可能会对当前用户有一定的影响\n",
    "    \"\"\"\n",
    "    nusers = np.shape(self.userFriends)[1]\n",
    "    i = self.userIndex[userId]\n",
    "    return (self.userFriends[i, :].sum(axis=0) / nusers)[0,0]\n",
    "\n",
    "  def eventPop(self, eventId):\n",
    "    \"\"\"\n",
    "    本活动本身的热度\n",
    "    主要是通过参与的人数来界定的\n",
    "    \"\"\"\n",
    "    i = self.eventIndex[eventId]\n",
    "    return self.eventPopularity[i, 0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generateRSData(RS, train=True, header=True):\n",
    "    \"\"\"\n",
    "    把前面user-based协同过滤 和 item-based协同过滤，以及各种热度和影响度作为特征组合在一起\n",
    "    生成新的训练数据，用于分类器分类使用\n",
    "    \"\"\"\n",
    "    fn = \"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv\" if train else \"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/test.csv\"\n",
    "    fin = open(fn, 'rb')\n",
    "    fout = open(\"C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/\" + \"RS_\" + \"train.csv\", 'wb')\n",
    "    \n",
    "    #忽略第一行（列名字）\n",
    "    fin.readline().decode('utf8').strip().split(\",\")\n",
    "    \n",
    "    # write output header\n",
    "    if header:\n",
    "      ocolnames = [\"invited\", \"userCF_reco\", \"evtCF_reco\",\"svdCF_reco\",\"user_reco\", \"evt_p_reco\",\n",
    "        \"evt_c_reco\", \"user_pop\", \"frnd_infl\", \"evt_pop\"]\n",
    "      if train:\n",
    "        ocolnames.append(\"interested\")\n",
    "        ocolnames.append(\"not_interested\")\n",
    "      fout.write(\",\".join(ocolnames).encode('utf8') + b\"\\n\")\n",
    "    \n",
    "    ln = 0\n",
    "    for line in fin:\n",
    "      ln += 1\n",
    "      if ln%500 == 0:\n",
    "          print(\"%s:%d (userId, eventId)=(%s, %s)\" % (fn, ln, userId, eventId))\n",
    "          #break;\n",
    "      \n",
    "      cols = line.decode('utf8').strip().split(\",\")\n",
    "      userId = cols[0]\n",
    "      eventId = cols[1]\n",
    "      invited = cols[2]\n",
    "      \n",
    "      #协同过滤推荐\n",
    "      userCF_reco = RS.userCFReco(userId, eventId)\n",
    "      itemCF_reco = RS.eventCFReco(userId, eventId)\n",
    "      svdCF_reco = RS.svdCFReco(userId, eventId)\n",
    "      \n",
    "      #基于用户属性相似度的推荐\n",
    "      user_reco = RS.userReco(userId, eventId)\n",
    "    \n",
    "      #基于活动属性相似度的推荐\n",
    "      evt_p_reco, evt_c_reco = RS.eventReco(userId, eventId)\n",
    "    \n",
    "      #基于用户的朋友个数来推断用户的社交程度\n",
    "      user_pop = RS.userPop(userId)\n",
    "     \n",
    "      #基于用户社交属性的推荐\n",
    "      frnd_infl = RS.friendInfluence(userId)\n",
    "        \n",
    "      #基于活动热度的推荐  \n",
    "      evt_pop = RS.eventPop(eventId)\n",
    "        \n",
    "      #所有推荐度串联  \n",
    "      ocols = [invited, userCF_reco, itemCF_reco, svdCF_reco,user_reco, evt_p_reco,\n",
    "        evt_c_reco, user_pop, frnd_infl, evt_pop]\n",
    "      \n",
    "      if train:\n",
    "        ocols.append(cols[4]) # interested\n",
    "        ocols.append(cols[5]) # not_interested\n",
    "      fout.write(\",\".join(map(lambda x: str(x), ocols)).encode('utf8') + b\"\\n\")\n",
    "    \n",
    "    fin.close()\n",
    "    fout.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVD Train...\n",
      "SVD trained\n",
      "生成训练数据...\n",
      "\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:500 (userId, eventId)=(123290209, 1887085024)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:1000 (userId, eventId)=(272886293, 199858305)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:1500 (userId, eventId)=(395305791, 1582270949)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:2000 (userId, eventId)=(527523423, 3272728211)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:2500 (userId, eventId)=(651258472, 792632006)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:3000 (userId, eventId)=(811791433, 524756826)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:3500 (userId, eventId)=(985547042, 1269035551)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:4000 (userId, eventId)=(1107615001, 173949238)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:4500 (userId, eventId)=(1236336671, 3849306291)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:5000 (userId, eventId)=(1414301782, 2652356640)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:5500 (userId, eventId)=(1595465532, 955398943)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:6000 (userId, eventId)=(1747091728, 2131379889)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:6500 (userId, eventId)=(1914182220, 955398943)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:7000 (userId, eventId)=(2071842684, 1076364848)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:7500 (userId, eventId)=(2217853337, 3051438735)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:8000 (userId, eventId)=(2338481531, 2525447278)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:8500 (userId, eventId)=(2489551967, 520657921)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:9000 (userId, eventId)=(2650493630, 87962584)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:9500 (userId, eventId)=(2791418962, 4223848259)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:10000 (userId, eventId)=(2903662804, 2791462807)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:10500 (userId, eventId)=(3036141956, 3929507420)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:11000 (userId, eventId)=(3176074542, 3459485614)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:11500 (userId, eventId)=(3285425249, 2271782630)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:12000 (userId, eventId)=(3410667855, 1063772489)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:12500 (userId, eventId)=(3531604778, 2584839423)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:13000 (userId, eventId)=(3686871863, 53495098)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:13500 (userId, eventId)=(3833637800, 2415873572)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:14000 (userId, eventId)=(3944021305, 2096772901)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:14500 (userId, eventId)=(4075466480, 3567240505)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/train.csv:15000 (userId, eventId)=(4197193550, 1628057176)\n",
      "生成预测数据...\n",
      "\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/test.csv:500 (userId, eventId)=(182290053, 2529072432)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/test.csv:1000 (userId, eventId)=(433510318, 4244463632)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/test.csv:1500 (userId, eventId)=(632808865, 2845303452)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/test.csv:2000 (userId, eventId)=(813611885, 2036538169)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/test.csv:2500 (userId, eventId)=(1010701404, 303459881)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/test.csv:3000 (userId, eventId)=(1210932037, 2529072432)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/test.csv:3500 (userId, eventId)=(1452921099, 2705317682)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/test.csv:4000 (userId, eventId)=(1623287180, 1626678328)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/test.csv:4500 (userId, eventId)=(1855201342, 2603032829)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/test.csv:5000 (userId, eventId)=(2083900381, 2529072432)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/test.csv:5500 (userId, eventId)=(2318415276, 2509151803)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/test.csv:6000 (userId, eventId)=(2528161539, 4025975316)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/test.csv:6500 (userId, eventId)=(2749110768, 4244406355)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/test.csv:7000 (userId, eventId)=(2927772127, 1532377761)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/test.csv:7500 (userId, eventId)=(3199685636, 1776393554)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/test.csv:8000 (userId, eventId)=(3393388475, 680270887)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/test.csv:8500 (userId, eventId)=(3601169721, 154434302)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/test.csv:9000 (userId, eventId)=(3828963415, 3067222491)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/test.csv:9500 (userId, eventId)=(4018723397, 2522610844)\n",
      "C:/Users/chenxi/Desktop/作业/第四周/第四周作业/homework4_作业说明/test.csv:10000 (userId, eventId)=(4180064266, 2658555390)\n"
     ]
    }
   ],
   "source": [
    "RS = RecommonderSystem()\n",
    "print(\"生成训练数据...\\n\")\n",
    "generateRSData(RS,train=True,  header=True)\n",
    "\n",
    "print(\"生成预测数据...\\n\")\n",
    "generateRSData(RS, train=False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
